from Opener import build_openers
import Parser
from datetime import datetime
from Configuration import DURATION, TIMEOUT, LITTLE_SLEEPING, ACTIVE_TIME
from Utility import loop_increase, emphasis_print, log_to_file
import time, random
import Dao
import urllib
import json


class Spider(object):

    def __init__(self, accounts):
        """

        :param accounts: a list of account objects
        """
        self.start_time = datetime.now()
        self.end_time = None

        self.accounts = accounts
        self.openers = build_openers(accounts)
        self.current_opener = 0
        self.parser = Parser

    def get_html(self, url):
        """

        :return: the html of specified url
        """
        opener = self.get_current_opener()
        html = None
        while html is None:
            try:
                html = opener.open(url, timeout=TIMEOUT).read()
            except Exception as e:
                print e.message, 'get_html().'

        return html

    def little_sleep(self):
        """
        Sleep between every page fetching.
        :return:
        """
        time.sleep(random.randint(LITTLE_SLEEPING, 2 * LITTLE_SLEEPING))

    def get_current_opener(self):
        """

        :return:
        """
        self.end_time = datetime.now()
        duration = (self.end_time - self.start_time).seconds
        if duration > DURATION:
            self.current_opener = loop_increase(self.current_opener, len(self.openers))
            self.start_time = datetime.now()
            return self.openers[self.current_opener]
        else:
            return self.openers[self.current_opener]

    def ban_account(self):
        """

        :return:
        """
        url = 'http://sass.weibo.com/unfreeze'

        html = self.openers[self.current_opener].open(url, timeout=TIMEOUT).read()

        is_exceptional = self.parser.is_exceptional(html)
        is_frozen = self.parser.is_frozen(html)
        if is_exceptional is False and is_frozen is False:
            return

        email = self.accounts[self.current_opener].email

        Dao.Account.ban_account(email)

        emphasis_print('%s has been banned!!!' % (email,))

        self.accounts.pop(self.current_opener)
        self.openers.pop(self.current_opener)
        if self.current_opener == len(self.openers):
            self.current_opener = 0

        if len(self.openers) == 0:
            raise Exception('No valid account!')

    def reset_account(self):

        email = self.accounts[self.current_opener].email
        Dao.Account.reset(email)
        emphasis_print('%s expires!!!' % (email,))

        self.accounts.pop(self.current_opener)
        self.openers.pop(self.current_opener)
        if self.current_opener == len(self.openers):
            self.current_opener = 0

        if len(self.openers) == 0:
            raise Exception('No valid account!')

    def detect_dirty_html(self, html):
        """
        Detect dirty html, if found, return true.
        :param html:
        :return:
        """
        new_uid = self.parser.parse_uid(html)
        if new_uid == -1:  # account may be banned.
            self.ban_account()
            return True

        is_visitor = self.parser.is_visitor(html)
        if is_visitor is True:
            self.reset_account()
            return True
        return False


class SeedSpider(Spider):

    def __init__(self, accounts, uid_list):
        """

        :param accounts: a list of account objects
        :param uid_list:
        """
        super(SeedSpider, self).__init__(accounts)
        self.seeds = uid_list
        self.checked_seeds = []
        self.followees = []

    def get_followee_page_num(self, html):
        """
        get the number of followee pages, five is the upper bound.
        :param html:
        :return:
        """
        fee_pnum = self.parser.parse_followee_page_num(html)
        if fee_pnum is None:
            return None
        if fee_pnum > 6:
            fee_pnum = 5
        return fee_pnum

    def get_followees(self, uid):
        """

        :param uid: target uid
        :return: a list of followee object
        """
        home_page_url = 'http://www.weibo.com/u/%s' % (uid,)
        home_page = self.get_html(home_page_url)
        pid = self.parser.parse_pid(home_page)  # get pid first

        url = 'http://www.weibo.com/p/' + pid + '/follow?from=page_' + pid[:6] + '&wvr=6&mod=headfollow#place'
        html = None
        while True:
            html = self.get_html(url)
            self.little_sleep()

            if self.detect_dirty_html(html) is True:
                continue

            fee_page_num = self.get_followee_page_num(html)
            if fee_page_num is not None:
                break
            else:
                print 'Cannot get followee page number -- pid: %s' % (pid,)
                continue

        if fee_page_num == 0:
            print 'He/She does not follow any one.'
            return
        else:
            print 'Getting followee page 1 of %d...' % (fee_page_num,)

            followees = self.parser.parse_followees(html, pid)
            self.followees.extend(followees)
            if fee_page_num == 1:
                return  # just one page

            for i in xrange(2, fee_page_num + 1):
                while True:
                    url = 'http://www.weibo.com/p/%s/follow?from=page_%s&wvr=6&mod=headfollow&page=%d#place'\
                          % (pid, pid[:6], i)
                    print 'Getting followee page %d of %d...' % (i, fee_page_num)
                    html = self.get_html(url)
                    self.little_sleep()

                    followees = self.parser.parse_followees(html, pid)
                    if followees is None:  # dirty html
                        print 'Cannot parse followee page correctly - pid:%s' % (pid,)
                        with open('log.txt', 'w') as writer:
                            writer.write(html)
                        continue
                    self.followees.extend(followees)
                    break

    def save(self):
        """
        Save seed followees into database.
        :return:
        """
        db = Dao.Database()
        db.connect()
        for fee in self.followees:
            db.session.merge(fee)
        db.close()
        self.followees = []

    def get_new_batch(self):
        self.seeds = []
        self.seeds = Dao.Seed.get_seeds()
        self.checked_seeds = []

    def run(self):
        """
        Keep crawling followees.
        :return:
        """
        while True:
            for sd in self.seeds:
                print 'Target UID: %s...' % (sd,)
                self.get_followees(sd)
                self.save()
                print 'Finished UID: %s...' % (sd,)
                self.checked_seeds.append(sd)
            print 'Getting new batch of seeds...'
            self.get_new_batch()

    def clean_up(self):
        self.save()

        unchecked_seeds = []
        for sd in self.seeds:
            if sd not in self.checked_seeds:
                unchecked_seeds.append(sd)
        Dao.Seed.reset_seeds(unchecked_seeds)
        Dao.Account.reset_accounts(self.accounts)


class ExtensionSpider(Spider):

    def __init__(self, accounts, uid_list):
        """

        :param accounts:
        :param uid_list:
        """
        self.start_time = datetime.now()
        self.end_time = None

        super(ExtensionSpider, self).__init__(accounts)
        self.sources = uid_list  # source weibo users
        self.checked = []
        self.results = []

    def check_activeness(self, uid):
        """
        Check the user's activeness
        :param uid: the user's id
        :return: Result User obj or None if not active
        """
        html = self.get_latest_timeline(uid)
        if html is None:
            return None

        result_user = self.parser.parse_latest_timeline(uid, html)
        if result_user is None:
            return None

        duration = result_user['timestamp'] - datetime.strptime(result_user['latest_tmln_time'], '%Y-%m-%d %H:%M')
        if duration.days > ACTIVE_TIME:
            return None
        return Dao.ResultUser(result_user)

    def save(self):
        db = Dao.Database()
        db.connect()

        for res in self.results:
            db.session.merge(res)

        db.close()
        self.results = []
        Dao.SeedFollowee.mark(self.checked)

    def get_new_batch(self):
        self.sources = Dao.SeedFollowee.get_seed_followees()
        self.checked = []

    def check_active_users(self):
        """
        Check the users in self.sources, if it's active, keep it; or discard it.
        :return:
        """
        while True:
            for src in self.sources:
                print 'Check activeness for UID: %s...' % (src,)
                result = self.check_activeness(src)
                if result is not None:
                    print 'Active!'
                    self.results.append(result)
                else:
                    print 'Negative!'
                self.checked.append(src)
            print 'Dumping into database...'
            self.save()
            print 'Getting new batch of seed followees...'
            self.get_new_batch()

    def get_latest_timeline(self, uid):
        # uid = '2373295690'  # debug
        body = {
            '__rnd': 1343647638078,
            '_k': 1343647471134109,
            '_t': 0,
            'count': 15,
            'end_id': 3473519214542343,
            'max_id': 3473279479126179,
            'page': 1,
            'pagebar': 1,
            'pre_page': 1,
            'uid': uid
        }
        body['page'] = 1
        body['count'] = '50'
        body['pagebar'] = ''
        body['pre_page'] = 0

        url = 'http://weibo.com/aj/mblog/mbloglist?' + urllib.urlencode(body)
        while True:
            try:
                jsn_data = self.get_html(url)
                self.little_sleep()
                if self.parser.parse_exceptional_user(jsn_data) is True:
                    return None

                data = json.loads(jsn_data)
                html = data['data']
                if u'WB_feed_type SW_fun S_line2' in html:
                    return html
                else:
                    return None  # no post.
            except Exception as e:
                print e
                test_url = 'http://www.weibo.com/u/%s' % (uid,)
                test_html = self.get_html(test_url)
                self.detect_dirty_html(test_html)

    def clean_up(self):
        self.save()

        Dao.Account.reset_accounts(self.accounts)

        unchecked = []
        for src in self.sources:
            if src not in self.checked:
                unchecked.append(src)

        Dao.SeedFollowee.reset_seed_fees(unchecked)


